#! /usr/bin/env bash

get_stock_list()
{
    for l in {0..2062..200}; do
        f=$(( l + 1 ))-$(( l + 200 ))
        echo curl -s -o rediff/nseall_$f.html "http://money.rediff.com/companies/nseall/$f"
    done
}

parse_stock_list()
{
    for f in $(ls -1 rediff/*.html); do
        grep -oE '<td>[^<> ]+</td>[[:blank:]]*$' "$f" | sed -r 's;</?td>;;g'
    done
}

get_google_stock_pages()
{
    while read s; do
        printf "$s: in progress"
        curl -s -o google/$s.html "https://www.google.com/finance?q=NSE%3A$s"
        printf "\r$s: done\n"
    done < $1
}

get_reuters_stock_pages()
{
    for f in $(ls -1 google/*.html); do
        printf "$f: in progress"
        query=$(grep -oE -m1 -e 'rpc%3D66%26symbol%3D.+\.NS' $f | sed -e 's/%3D/=/g' -e 's/%26/\&/g')
        curl -s -o reuters/${f##*/} "http://www.reuters.com/finance/stocks/financialHighlights?$query"
        printf "\r$f: done\n"
    done
}

convert_to_text()
{
    for f in $(ls -1 reuters/*.html); do
        printf "$f: in progress"
        html2text -ascii -nobs -o ${f%.*}.txt $f
        printf "\r$f: done\n"
    done
}

# get_stock_list
# parse_stock_list | sort >stocks-list.txt
# get_google_stock_pages stocks-list.txt
# get_reuters_stock_pages
# convert_to_text
